import scrapy
from scrapy import Request

from ..items import NovelItem


class NovelsSpider(scrapy.Spider):
    name = 'novels'
    allowed_domains = ['http://www.qishus.com/']
    start_urls = ['http://www.qishus.com/html/bestlist-1.html']
    index = 0

    def parse(self, response):
        rel = response.css('#mainlistUL')
        print("===========================", response)
        novel_list = rel.xpath('./li/span/a')
        url_list = []
        for a in novel_list:
            self.index += 1
            title = a.xpath('./@title').extract_first()
            url = 'https://xiazai.xqishu.com/txt/' + title.strip() + '.txt'
            url_list.append(url)
        item = NovelItem()
        item['file_urls'] = url_list
        # item['book_name'] = title
        yield item

        if self.index <= 120:
            rel = response.css('#showNextPage')
            a_list = rel.xpath('//p/a')
            for next_a in a_list:
                if next_a.xpath('./text()').extract_first() == '下一页':
                    next_url = next_a.xpath('./@href').extract_first()
                    next_url = response.urljoin(next_url)
                    print("ooooooooooooooooooooooo", next_url)
                    yield Request(next_url, callback=self.parse,dont_filter=True)
